{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Copyright (c) 2020, NVIDIA CORPORATION.\n",
    "Licensed under the Apache License, Version 2.0 (the \"License\");\n",
    "you may not use this file except in compliance with the License.\n",
    "You may obtain a copy of the License at\n",
    "    http://www.apache.org/licenses/LICENSE-2.0\n",
    "Unless required by applicable law or agreed to in writing, software\n",
    "distributed under the License is distributed on an \"AS IS\" BASIS,\n",
    "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
    "See the License for the specific language governing permissions and\n",
    "limitations under the License."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm.notebook import tqdm \n",
    "import glob\n",
    "import gc\n",
    "import os.path\n",
    "\n",
    "import hashlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_memory( df ):\n",
    "    features = df.columns\n",
    "    for i in range( df.shape[1] ):\n",
    "        if df.dtypes[i] == 'uint8':\n",
    "            df[features[i]] = df[features[i]].astype( np.int8 )\n",
    "            gc.collect()\n",
    "        elif df.dtypes[i] == 'bool':\n",
    "            df[features[i]] = df[features[i]].astype( np.int8 )\n",
    "            gc.collect()\n",
    "        elif df.dtypes[i] == 'uint32':\n",
    "            df[features[i]] = df[features[i]].astype( np.int32 )\n",
    "            gc.collect()\n",
    "        elif df.dtypes[i] == 'int64':\n",
    "            df[features[i]] = df[features[i]].astype( np.int32 )\n",
    "            gc.collect()\n",
    "        elif df.dtypes[i] == 'float64':\n",
    "            df[features[i]] = df[features[i]].astype( np.float32 )\n",
    "            gc.collect()\n",
    "    print( df.dtypes )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_hash(text, split_text='@', no=0):\n",
    "    text = text.lower()\n",
    "    uhash = ''\n",
    "    text_split = text.split('@')\n",
    "    if len(text_split)>(no+1):\n",
    "        text_split = text_split[no+1].split(' ')\n",
    "        cl_loop = True\n",
    "        uhash += clean_text(text_split[0])\n",
    "        while cl_loop:\n",
    "            if len(text_split)>1:\n",
    "                if text_split[1] in ['_']:\n",
    "                    uhash += clean_text(text_split[1]) + clean_text(text_split[2])\n",
    "                    text_split = text_split[2:]\n",
    "                else:\n",
    "                    cl_loop = False\n",
    "            else:\n",
    "                cl_loop = False\n",
    "    hash_object = hashlib.md5(uhash.encode('utf-8'))\n",
    "    return hash_object.hexdigest()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_text(text):\n",
    "    if len(text)>1:\n",
    "        if text[-1] in ['!', '?', ':', ';', '.', ',']:\n",
    "            return(text[:-1])\n",
    "    return(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((121386431, 3), (12434735, 3), (12434838, 3))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_parquet( 'train-tweet-1.parquet' )\n",
    "test0 = pd.read_parquet( 'test0-tweet-1.parquet' )\n",
    "test1 = pd.read_parquet( 'test1-tweet-1.parquet' )\n",
    "train.shape, test0.shape, test1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#train = train.head(100000)\n",
    "#test0 = test0.head(100000)\n",
    "#test1 = test1.head(100000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 12min 35s, sys: 55.8 s, total: 13min 31s\n",
      "Wall time: 13min 31s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "WORDS = {}\n",
    "DF = []\n",
    "for tweet in train['tweet'].unique():\n",
    "    words = tweet.split(' ')\n",
    "    for w in words:\n",
    "        if w not in WORDS:\n",
    "            WORDS[w] = 1\n",
    "        else:\n",
    "            WORDS[w]+= 1\n",
    "gc.collect()\n",
    "\n",
    "for tweet in test0['tweet'].unique():\n",
    "    words = tweet.split(' ')\n",
    "    for w in words:\n",
    "        if w not in WORDS:\n",
    "            WORDS[w] = 1\n",
    "        else:\n",
    "            WORDS[w]+= 1\n",
    "gc.collect()\n",
    "for tweet in test1['tweet'].unique():\n",
    "    words = tweet.split(' ')\n",
    "    for w in words:\n",
    "        if w not in WORDS:\n",
    "            WORDS[w] = 1\n",
    "        else:\n",
    "            WORDS[w]+= 1\n",
    "gc.collect()\n",
    "                \n",
    "len(WORDS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 11s, sys: 6.22 s, total: 1min 17s\n",
      "Wall time: 1min 17s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "count=0\n",
    "for w in WORDS:\n",
    "    WORDS[w] = [ WORDS[w], count ]\n",
    "    count+=1\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[2293, 205667]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "WORDS['marvel']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freq_encode_words( vs ):\n",
    "    li=[]\n",
    "    lf=[]\n",
    "    for v in vs.split(' '):\n",
    "        if v not in ['','[',']','.','!','@','_','#']:\n",
    "            f,i = WORDS[v]\n",
    "            if f<100000:\n",
    "                if f>2:\n",
    "                    li.append( str(i) )\n",
    "                    #li.append( v )\n",
    "                    lf.append( f )\n",
    "    return ' '.join( list((np.array(li)[np.argsort(lf)] )) )    \n",
    "    \n",
    "#freq_encode_words( train['tweet'].values[191019] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ret_word( x, rw=0 ):\n",
    "    x = x.split(' ')\n",
    "    if rw==0:\n",
    "        if len(x)>=1:\n",
    "            return x[0]\n",
    "    elif rw==1:\n",
    "        if len(x)>=2:\n",
    "            return x[1]\n",
    "    elif rw== -1:\n",
    "        if len(x)>=1:\n",
    "            return x[-1]\n",
    "    elif rw== -2:\n",
    "        if len(x)>=2:\n",
    "            return x[-2]\n",
    "\n",
    "    return '-1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id                 int32\n",
      "count_ats          int32\n",
      "count_char         int32\n",
      "count_words        int32\n",
      "hash0             object\n",
      "hash1             object\n",
      "tw_uhash          object\n",
      "tw_hash            int32\n",
      "tw_freq_hash       int32\n",
      "tw_first_word     object\n",
      "tw_second_word    object\n",
      "tw_last_word      object\n",
      "tw_llast_word     object\n",
      "tw_len             int32\n",
      "dtype: object\n",
      "CPU times: user 2h 19min 29s, sys: 2h 13min 56s, total: 4h 33min 25s\n",
      "Wall time: 4h 31min 43s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "DF = []\n",
    "train['tweet_nortsign'] = train['tweet'].str.replace('\\[CLS\\] RT @', '')\n",
    "train['count_words']    = train['tweet'].str.count(' ')\n",
    "train['count_char']     = train['tweet'].apply(lambda x: len(x))\n",
    "train['count_ats']      = train['tweet_nortsign'].str.count('@')\n",
    "train['hash0']          = train['tweet_nortsign'].apply(lambda x: extract_hash(x))\n",
    "train['hash1']          = train['tweet_nortsign'].apply(lambda x: extract_hash(x, no=1))\n",
    "train['tw_uhash']       = train['tweet'].apply(lambda x: extract_hash(x, split_text='RT @', no=0))\n",
    "train['tw_hash']        = train['tweet'].apply(lambda x: hash(x)%1000000000 )\n",
    "\n",
    "train['tweet']          = train['tweet'].apply(lambda x: freq_encode_words(x) )\n",
    "train['tw_freq_hash']   = train['tweet'].apply(lambda x: hash(x)%1000000000 )\n",
    "train['tw_first_word']  = train['tweet'].apply(lambda x: ret_word(x,0) )\n",
    "train['tw_second_word'] = train['tweet'].apply(lambda x: ret_word(x,1) )\n",
    "train['tw_last_word']   = train['tweet'].apply(lambda x: ret_word(x,-1) )\n",
    "train['tw_llast_word']  = train['tweet'].apply(lambda x: ret_word(x,-2) )\n",
    "train['tw_len']         = train['tweet'].apply(lambda x: len(x.split(' ')) )\n",
    "\n",
    "DF.append( train[['id','count_ats', 'count_char', 'count_words', 'hash0', 'hash1', 'tw_uhash','tw_hash','tw_freq_hash','tw_first_word','tw_second_word','tw_last_word','tw_llast_word','tw_len']] )\n",
    "del train\n",
    "gc.collect()\n",
    "    \n",
    "\n",
    "test0['tweet_nortsign'] = test0['tweet'].str.replace('\\[CLS\\] RT @', '')\n",
    "test0['count_words']    = test0['tweet'].str.count(' ')\n",
    "test0['count_char']     = test0['tweet'].apply(lambda x: len(x))\n",
    "test0['count_ats']      = test0['tweet_nortsign'].str.count('@')\n",
    "test0['hash0']          = test0['tweet_nortsign'].apply(lambda x: extract_hash(x))\n",
    "test0['hash1']          = test0['tweet_nortsign'].apply(lambda x: extract_hash(x, no=1))\n",
    "test0['tw_uhash']       = test0['tweet'].apply(lambda x: extract_hash(x, split_text='RT @', no=0))\n",
    "test0['tw_hash']        = test0['tweet'].apply(lambda x: hash(x)%1000000000 )\n",
    "\n",
    "test0['tweet']          = test0['tweet'].apply(lambda x: freq_encode_words(x) )\n",
    "test0['tw_freq_hash']   = test0['tweet'].apply(lambda x: hash(x)%1000000000 )\n",
    "test0['tw_first_word']  = test0['tweet'].apply(lambda x: ret_word(x,0) )\n",
    "test0['tw_second_word'] = test0['tweet'].apply(lambda x: ret_word(x,1) )\n",
    "test0['tw_last_word']   = test0['tweet'].apply(lambda x: ret_word(x,-1) )\n",
    "test0['tw_llast_word']  = test0['tweet'].apply(lambda x: ret_word(x,-2) )\n",
    "test0['tw_len']         = test0['tweet'].apply(lambda x: len(x.split(' ')) )\n",
    "\n",
    "DF.append( test0[['id','count_ats', 'count_char', 'count_words', 'hash0', 'hash1', 'tw_uhash','tw_hash','tw_freq_hash','tw_first_word','tw_second_word','tw_last_word','tw_llast_word','tw_len']] )\n",
    "del test0\n",
    "gc.collect()\n",
    "    \n",
    "\n",
    "test1['tweet_nortsign'] = test1['tweet'].str.replace('\\[CLS\\] RT @', '')\n",
    "test1['count_words']    = test1['tweet'].str.count(' ')\n",
    "test1['count_char']     = test1['tweet'].apply(lambda x: len(x))\n",
    "test1['count_ats']      = test1['tweet_nortsign'].str.count('@')\n",
    "test1['hash0']          = test1['tweet_nortsign'].apply(lambda x: extract_hash(x))\n",
    "test1['hash1']          = test1['tweet_nortsign'].apply(lambda x: extract_hash(x, no=1))\n",
    "test1['tw_uhash']       = test1['tweet'].apply(lambda x: extract_hash(x, split_text='RT @', no=0))\n",
    "test1['tw_hash']        = test1['tweet'].apply(lambda x: hash(x)%1000000000 )\n",
    "\n",
    "test1['tweet']          = test1['tweet'].apply(lambda x: freq_encode_words(x) )\n",
    "test1['tw_freq_hash']   = test1['tweet'].apply(lambda x: hash(x)%1000000000 )\n",
    "test1['tw_first_word']  = test1['tweet'].apply(lambda x: ret_word(x,0) )\n",
    "test1['tw_second_word'] = test1['tweet'].apply(lambda x: ret_word(x,1) )\n",
    "test1['tw_last_word']   = test1['tweet'].apply(lambda x: ret_word(x,-1) )\n",
    "test1['tw_llast_word']  = test1['tweet'].apply(lambda x: ret_word(x,-2) )\n",
    "test1['tw_len']         = test1['tweet'].apply(lambda x: len(x.split(' ')) )\n",
    "\n",
    "DF.append( test1[['id','count_ats', 'count_char', 'count_words', 'hash0', 'hash1', 'tw_uhash','tw_hash','tw_freq_hash','tw_first_word','tw_second_word','tw_last_word','tw_llast_word','tw_len']] )\n",
    "del test1\n",
    "gc.collect()\n",
    "\n",
    "\n",
    "DF = pd.concat( DF )\n",
    "gc.collect()\n",
    "\n",
    "save_memory( DF )\n",
    "DF = DF.reset_index( drop=True )\n",
    "gc.collect()\n",
    "#DF.to_parquet( '../input/text-processings-1.parquet' )\n",
    "DF.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(6690786, 4)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>level_0</th>\n",
       "      <th>index</th>\n",
       "      <th>0</th>\n",
       "      <th>uid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>347428021</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>9b9672de2cbe5ddd7ebd7538945a970a</td>\n",
       "      <td>1294764</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>b14a7b8059d9c055954c92674ce60032</td>\n",
       "      <td>976238</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>31cf2397511c7cfce33506bef80e25b7</td>\n",
       "      <td>660605</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>ba9bf05693b9fa202d922dd43a08f281</td>\n",
       "      <td>194430</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   level_0                             index          0  uid\n",
       "0        0  d41d8cd98f00b204e9800998ecf8427e  347428021    0\n",
       "1        1  9b9672de2cbe5ddd7ebd7538945a970a    1294764    1\n",
       "2        2  b14a7b8059d9c055954c92674ce60032     976238    2\n",
       "3        3  31cf2397511c7cfce33506bef80e25b7     660605    3\n",
       "4        4  ba9bf05693b9fa202d922dd43a08f281     194430    4"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uhashes = pd.concat([DF['hash0'], DF['hash1'], DF['tw_uhash']], axis=0)\n",
    "gc.collect()\n",
    "uhashes = uhashes.value_counts()\n",
    "gc.collect()\n",
    "uhashes = uhashes.reset_index().reset_index()\n",
    "gc.collect()\n",
    "uhashes['uid'] = np.arange(0,uhashes.shape[0] )\n",
    "print( uhashes.shape )\n",
    "uhashes.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>count_ats</th>\n",
       "      <th>count_char</th>\n",
       "      <th>count_words</th>\n",
       "      <th>hash0</th>\n",
       "      <th>hash1</th>\n",
       "      <th>tw_uhash</th>\n",
       "      <th>tw_hash</th>\n",
       "      <th>tw_freq_hash</th>\n",
       "      <th>tw_first_word</th>\n",
       "      <th>tw_second_word</th>\n",
       "      <th>tw_last_word</th>\n",
       "      <th>tw_llast_word</th>\n",
       "      <th>tw_len</th>\n",
       "      <th>tw_hash0</th>\n",
       "      <th>tw_hash1</th>\n",
       "      <th>tw_rt_uhash</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>68</td>\n",
       "      <td>17</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>417904705</td>\n",
       "      <td>14077494</td>\n",
       "      <td>9</td>\n",
       "      <td>16</td>\n",
       "      <td>15</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>182</td>\n",
       "      <td>37</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>0151f99e5c67748adcbb0c694d7e0b1d</td>\n",
       "      <td>254921337</td>\n",
       "      <td>594291702</td>\n",
       "      <td>24</td>\n",
       "      <td>19</td>\n",
       "      <td>36</td>\n",
       "      <td>21</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>661291</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>105</td>\n",
       "      <td>24</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>917286447</td>\n",
       "      <td>46081915</td>\n",
       "      <td>58</td>\n",
       "      <td>52</td>\n",
       "      <td>57</td>\n",
       "      <td>59</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>103</td>\n",
       "      <td>22</td>\n",
       "      <td>bd562f502b23a877c219369fc82e1dd2</td>\n",
       "      <td>73fa9240c65d0231cfca427bb6b9dc88</td>\n",
       "      <td>184b3f811e4e3f2eb9d763cd33dffdf6</td>\n",
       "      <td>456659357</td>\n",
       "      <td>983140245</td>\n",
       "      <td>65</td>\n",
       "      <td>64</td>\n",
       "      <td>76</td>\n",
       "      <td>75</td>\n",
       "      <td>10</td>\n",
       "      <td>198539</td>\n",
       "      <td>2048</td>\n",
       "      <td>616</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>237</td>\n",
       "      <td>63</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>559483437</td>\n",
       "      <td>636979676</td>\n",
       "      <td>129</td>\n",
       "      <td>101</td>\n",
       "      <td>89</td>\n",
       "      <td>117</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>124</td>\n",
       "      <td>20</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>457368886</td>\n",
       "      <td>984799237</td>\n",
       "      <td>138</td>\n",
       "      <td>132</td>\n",
       "      <td>131</td>\n",
       "      <td>135</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>79</td>\n",
       "      <td>11</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>564036868</td>\n",
       "      <td>94306893</td>\n",
       "      <td>154</td>\n",
       "      <td>148</td>\n",
       "      <td>152</td>\n",
       "      <td>150</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>304</td>\n",
       "      <td>52</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>565022653</td>\n",
       "      <td>780989501</td>\n",
       "      <td>172</td>\n",
       "      <td>168</td>\n",
       "      <td>192</td>\n",
       "      <td>169</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>94</td>\n",
       "      <td>12</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>46c2dda06911cbfb91b3291026e41e31</td>\n",
       "      <td>807354930</td>\n",
       "      <td>233871987</td>\n",
       "      <td>196</td>\n",
       "      <td>198</td>\n",
       "      <td>200</td>\n",
       "      <td>201</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4983</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>98</td>\n",
       "      <td>11</td>\n",
       "      <td>c17c54808d0e99a9abfc83e3d10dad0b</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>c17c54808d0e99a9abfc83e3d10dad0b</td>\n",
       "      <td>963395355</td>\n",
       "      <td>711673564</td>\n",
       "      <td>211</td>\n",
       "      <td>207</td>\n",
       "      <td>208</td>\n",
       "      <td>206</td>\n",
       "      <td>5</td>\n",
       "      <td>1738</td>\n",
       "      <td>0</td>\n",
       "      <td>1738</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>192</td>\n",
       "      <td>24</td>\n",
       "      <td>5683535250516ef6b17c17f2e65721e1</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>5683535250516ef6b17c17f2e65721e1</td>\n",
       "      <td>930511994</td>\n",
       "      <td>30642644</td>\n",
       "      <td>228</td>\n",
       "      <td>226</td>\n",
       "      <td>217</td>\n",
       "      <td>227</td>\n",
       "      <td>15</td>\n",
       "      <td>36</td>\n",
       "      <td>0</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>166</td>\n",
       "      <td>23</td>\n",
       "      <td>5f9a9fbbb0694a3d2d5b3cf06579be4f</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>5f9a9fbbb0694a3d2d5b3cf06579be4f</td>\n",
       "      <td>375576056</td>\n",
       "      <td>228880814</td>\n",
       "      <td>231</td>\n",
       "      <td>233</td>\n",
       "      <td>234</td>\n",
       "      <td>235</td>\n",
       "      <td>7</td>\n",
       "      <td>36064</td>\n",
       "      <td>0</td>\n",
       "      <td>36064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>91</td>\n",
       "      <td>9</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>744776734</td>\n",
       "      <td>222199672</td>\n",
       "      <td>252</td>\n",
       "      <td>251</td>\n",
       "      <td>248</td>\n",
       "      <td>250</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>129</td>\n",
       "      <td>15</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>374887941</td>\n",
       "      <td>146983276</td>\n",
       "      <td>265</td>\n",
       "      <td>266</td>\n",
       "      <td>255</td>\n",
       "      <td>254</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>14</td>\n",
       "      <td>0</td>\n",
       "      <td>268</td>\n",
       "      <td>77</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>846154718</td>\n",
       "      <td>271081953</td>\n",
       "      <td>293</td>\n",
       "      <td>285</td>\n",
       "      <td>269</td>\n",
       "      <td>59</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>61</td>\n",
       "      <td>10</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>714654354</td>\n",
       "      <td>296228343</td>\n",
       "      <td>323</td>\n",
       "      <td>329</td>\n",
       "      <td>326</td>\n",
       "      <td>325</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>71</td>\n",
       "      <td>13</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>570244015</td>\n",
       "      <td>70009288</td>\n",
       "      <td>338</td>\n",
       "      <td>339</td>\n",
       "      <td>336</td>\n",
       "      <td>331</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>17</td>\n",
       "      <td>2</td>\n",
       "      <td>261</td>\n",
       "      <td>36</td>\n",
       "      <td>9e69525fced19ff56413ff74b6975b92</td>\n",
       "      <td>9c77bb178ebef995c63c5ed56a89cee6</td>\n",
       "      <td>9e69525fced19ff56413ff74b6975b92</td>\n",
       "      <td>615370178</td>\n",
       "      <td>959196222</td>\n",
       "      <td>344</td>\n",
       "      <td>362</td>\n",
       "      <td>353</td>\n",
       "      <td>358</td>\n",
       "      <td>21</td>\n",
       "      <td>23431</td>\n",
       "      <td>10141</td>\n",
       "      <td>23431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>155</td>\n",
       "      <td>24</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>ac85de223a3ab28d600324374d0e1cd9</td>\n",
       "      <td>605457715</td>\n",
       "      <td>588271767</td>\n",
       "      <td>377</td>\n",
       "      <td>365</td>\n",
       "      <td>374</td>\n",
       "      <td>370</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>39373</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "      <td>147</td>\n",
       "      <td>26</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>d41d8cd98f00b204e9800998ecf8427e</td>\n",
       "      <td>0bdb0007e1e6be128063d5c4e9a9f4e6</td>\n",
       "      <td>810627631</td>\n",
       "      <td>993602027</td>\n",
       "      <td>383</td>\n",
       "      <td>382</td>\n",
       "      <td>390</td>\n",
       "      <td>384</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>840</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    id  count_ats  count_char  count_words                             hash0  \\\n",
       "0    0          0          68           17  d41d8cd98f00b204e9800998ecf8427e   \n",
       "1    1          0         182           37  d41d8cd98f00b204e9800998ecf8427e   \n",
       "2    2          0         105           24  d41d8cd98f00b204e9800998ecf8427e   \n",
       "3    3          2         103           22  bd562f502b23a877c219369fc82e1dd2   \n",
       "4    4          0         237           63  d41d8cd98f00b204e9800998ecf8427e   \n",
       "5    5          0         124           20  d41d8cd98f00b204e9800998ecf8427e   \n",
       "6    6          0          79           11  d41d8cd98f00b204e9800998ecf8427e   \n",
       "7    7          0         304           52  d41d8cd98f00b204e9800998ecf8427e   \n",
       "8    8          0          94           12  d41d8cd98f00b204e9800998ecf8427e   \n",
       "9    9          1          98           11  c17c54808d0e99a9abfc83e3d10dad0b   \n",
       "10  10          1         192           24  5683535250516ef6b17c17f2e65721e1   \n",
       "11  11          1         166           23  5f9a9fbbb0694a3d2d5b3cf06579be4f   \n",
       "12  12          0          91            9  d41d8cd98f00b204e9800998ecf8427e   \n",
       "13  13          0         129           15  d41d8cd98f00b204e9800998ecf8427e   \n",
       "14  14          0         268           77  d41d8cd98f00b204e9800998ecf8427e   \n",
       "15  15          0          61           10  d41d8cd98f00b204e9800998ecf8427e   \n",
       "16  16          0          71           13  d41d8cd98f00b204e9800998ecf8427e   \n",
       "17  17          2         261           36  9e69525fced19ff56413ff74b6975b92   \n",
       "18  18          0         155           24  d41d8cd98f00b204e9800998ecf8427e   \n",
       "19  19          0         147           26  d41d8cd98f00b204e9800998ecf8427e   \n",
       "\n",
       "                               hash1                          tw_uhash  \\\n",
       "0   d41d8cd98f00b204e9800998ecf8427e  d41d8cd98f00b204e9800998ecf8427e   \n",
       "1   d41d8cd98f00b204e9800998ecf8427e  0151f99e5c67748adcbb0c694d7e0b1d   \n",
       "2   d41d8cd98f00b204e9800998ecf8427e  d41d8cd98f00b204e9800998ecf8427e   \n",
       "3   73fa9240c65d0231cfca427bb6b9dc88  184b3f811e4e3f2eb9d763cd33dffdf6   \n",
       "4   d41d8cd98f00b204e9800998ecf8427e  d41d8cd98f00b204e9800998ecf8427e   \n",
       "5   d41d8cd98f00b204e9800998ecf8427e  d41d8cd98f00b204e9800998ecf8427e   \n",
       "6   d41d8cd98f00b204e9800998ecf8427e  d41d8cd98f00b204e9800998ecf8427e   \n",
       "7   d41d8cd98f00b204e9800998ecf8427e  d41d8cd98f00b204e9800998ecf8427e   \n",
       "8   d41d8cd98f00b204e9800998ecf8427e  46c2dda06911cbfb91b3291026e41e31   \n",
       "9   d41d8cd98f00b204e9800998ecf8427e  c17c54808d0e99a9abfc83e3d10dad0b   \n",
       "10  d41d8cd98f00b204e9800998ecf8427e  5683535250516ef6b17c17f2e65721e1   \n",
       "11  d41d8cd98f00b204e9800998ecf8427e  5f9a9fbbb0694a3d2d5b3cf06579be4f   \n",
       "12  d41d8cd98f00b204e9800998ecf8427e  d41d8cd98f00b204e9800998ecf8427e   \n",
       "13  d41d8cd98f00b204e9800998ecf8427e  d41d8cd98f00b204e9800998ecf8427e   \n",
       "14  d41d8cd98f00b204e9800998ecf8427e  d41d8cd98f00b204e9800998ecf8427e   \n",
       "15  d41d8cd98f00b204e9800998ecf8427e  d41d8cd98f00b204e9800998ecf8427e   \n",
       "16  d41d8cd98f00b204e9800998ecf8427e  d41d8cd98f00b204e9800998ecf8427e   \n",
       "17  9c77bb178ebef995c63c5ed56a89cee6  9e69525fced19ff56413ff74b6975b92   \n",
       "18  d41d8cd98f00b204e9800998ecf8427e  ac85de223a3ab28d600324374d0e1cd9   \n",
       "19  d41d8cd98f00b204e9800998ecf8427e  0bdb0007e1e6be128063d5c4e9a9f4e6   \n",
       "\n",
       "      tw_hash  tw_freq_hash tw_first_word tw_second_word tw_last_word  \\\n",
       "0   417904705      14077494             9             16           15   \n",
       "1   254921337     594291702            24             19           36   \n",
       "2   917286447      46081915            58             52           57   \n",
       "3   456659357     983140245            65             64           76   \n",
       "4   559483437     636979676           129            101           89   \n",
       "5   457368886     984799237           138            132          131   \n",
       "6   564036868      94306893           154            148          152   \n",
       "7   565022653     780989501           172            168          192   \n",
       "8   807354930     233871987           196            198          200   \n",
       "9   963395355     711673564           211            207          208   \n",
       "10  930511994      30642644           228            226          217   \n",
       "11  375576056     228880814           231            233          234   \n",
       "12  744776734     222199672           252            251          248   \n",
       "13  374887941     146983276           265            266          255   \n",
       "14  846154718     271081953           293            285          269   \n",
       "15  714654354     296228343           323            329          326   \n",
       "16  570244015      70009288           338            339          336   \n",
       "17  615370178     959196222           344            362          353   \n",
       "18  605457715     588271767           377            365          374   \n",
       "19  810627631     993602027           383            382          390   \n",
       "\n",
       "   tw_llast_word  tw_len  tw_hash0  tw_hash1  tw_rt_uhash  \n",
       "0              8       7         0         0            0  \n",
       "1             21      12         0         0       661291  \n",
       "2             59       6         0         0            0  \n",
       "3             75      10    198539      2048          616  \n",
       "4            117      16         0         0            0  \n",
       "5            135      13         0         0            0  \n",
       "6            150       4         0         0            0  \n",
       "7            169      29         0         0            0  \n",
       "8            201       5         0         0         4983  \n",
       "9            206       5      1738         0         1738  \n",
       "10           227      15        36         0           36  \n",
       "11           235       7     36064         0        36064  \n",
       "12           250       5         0         0            0  \n",
       "13           254      10         0         0            0  \n",
       "14            59      30         0         0            0  \n",
       "15           325       7         0         0            0  \n",
       "16           331       4         0         0            0  \n",
       "17           358      21     23431     10141        23431  \n",
       "18           370       8         0         0        39373  \n",
       "19           384       9         0         0          840  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DF['tw_hash0']    = pd.merge( DF[['hash0']]  , uhashes[['index','uid']], left_on='hash0'  , right_on='index', how='left' )['uid']\n",
    "gc.collect()\n",
    "DF['tw_hash1']    = pd.merge( DF[['hash1']]  , uhashes[['index','uid']], left_on='hash1'  , right_on='index', how='left' )['uid']\n",
    "gc.collect()\n",
    "DF['tw_rt_uhash'] = pd.merge( DF[['tw_uhash']], uhashes[['index','uid']], left_on='tw_uhash', right_on='index', how='left' )['uid']\n",
    "gc.collect()\n",
    "DF.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id                 int32\n",
      "count_ats          int32\n",
      "count_char         int32\n",
      "count_words        int32\n",
      "tw_hash            int32\n",
      "tw_freq_hash       int32\n",
      "tw_first_word     object\n",
      "tw_second_word    object\n",
      "tw_last_word      object\n",
      "tw_llast_word     object\n",
      "tw_len             int32\n",
      "tw_hash0           int32\n",
      "tw_hash1           int32\n",
      "tw_rt_uhash        int32\n",
      "dtype: object\n"
     ]
    }
   ],
   "source": [
    "del DF['hash0'],DF['hash1'],DF['tw_uhash']\n",
    "gc.collect()\n",
    "save_memory( DF )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DF['tw_hash']        = pd.factorize( DF['tw_hash'] )[0]\n",
    "DF['tw_freq_hash']   = pd.factorize( DF['tw_freq_hash'] )[0]\n",
    "DF['tw_first_word']  = pd.factorize( DF['tw_first_word'] )[0]\n",
    "DF['tw_second_word'] = pd.factorize( DF['tw_second_word'] )[0]\n",
    "DF['tw_last_word']   = pd.factorize( DF['tw_last_word'] )[0]\n",
    "DF['tw_llast_word']  = pd.factorize( DF['tw_llast_word'] )[0]\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DF['tw_hash']        = DF['tw_hash'].astype(np.int32)\n",
    "DF['tw_freq_hash']   = DF['tw_freq_hash'].astype(np.int32)\n",
    "DF['tw_first_word']  = DF['tw_first_word'].astype(np.int32)\n",
    "DF['tw_second_word'] = DF['tw_second_word'].astype(np.int32)\n",
    "DF['tw_last_word']   = DF['tw_last_word'].astype(np.int32)\n",
    "DF['tw_llast_word']  = DF['tw_llast_word'].astype(np.int32)\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>count_ats</th>\n",
       "      <th>count_char</th>\n",
       "      <th>count_words</th>\n",
       "      <th>tw_hash</th>\n",
       "      <th>tw_freq_hash</th>\n",
       "      <th>tw_first_word</th>\n",
       "      <th>tw_second_word</th>\n",
       "      <th>tw_last_word</th>\n",
       "      <th>tw_llast_word</th>\n",
       "      <th>tw_len</th>\n",
       "      <th>tw_hash0</th>\n",
       "      <th>tw_hash1</th>\n",
       "      <th>tw_rt_uhash</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>68</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>182</td>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>661291</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>105</td>\n",
       "      <td>24</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>103</td>\n",
       "      <td>22</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>198539</td>\n",
       "      <td>2048</td>\n",
       "      <td>616</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>237</td>\n",
       "      <td>63</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>124</td>\n",
       "      <td>20</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>79</td>\n",
       "      <td>11</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>304</td>\n",
       "      <td>52</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>94</td>\n",
       "      <td>12</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4983</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>98</td>\n",
       "      <td>11</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>5</td>\n",
       "      <td>1738</td>\n",
       "      <td>0</td>\n",
       "      <td>1738</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  count_ats  count_char  count_words  tw_hash  tw_freq_hash  \\\n",
       "0   0          0          68           17        0             0   \n",
       "1   1          0         182           37        1             1   \n",
       "2   2          0         105           24        2             2   \n",
       "3   3          2         103           22        3             3   \n",
       "4   4          0         237           63        4             4   \n",
       "5   5          0         124           20        5             5   \n",
       "6   6          0          79           11        6             6   \n",
       "7   7          0         304           52        7             7   \n",
       "8   8          0          94           12        8             8   \n",
       "9   9          1          98           11        9             9   \n",
       "\n",
       "   tw_first_word  tw_second_word  tw_last_word  tw_llast_word  tw_len  \\\n",
       "0              0               0             0              0       7   \n",
       "1              1               1             1              1      12   \n",
       "2              2               2             2              2       6   \n",
       "3              3               3             3              3      10   \n",
       "4              4               4             4              4      16   \n",
       "5              5               5             5              5      13   \n",
       "6              6               6             6              6       4   \n",
       "7              7               7             7              7      29   \n",
       "8              8               8             8              8       5   \n",
       "9              9               9             9              9       5   \n",
       "\n",
       "   tw_hash0  tw_hash1  tw_rt_uhash  \n",
       "0         0         0            0  \n",
       "1         0         0       661291  \n",
       "2         0         0            0  \n",
       "3    198539      2048          616  \n",
       "4         0         0            0  \n",
       "5         0         0            0  \n",
       "6         0         0            0  \n",
       "7         0         0            0  \n",
       "8         0         0         4983  \n",
       "9      1738         0         1738  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DF.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DF.to_parquet( 'text-processings-1.parquet' )\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                int32\n",
       "count_ats         int32\n",
       "count_char        int32\n",
       "count_words       int32\n",
       "tw_hash           int32\n",
       "tw_freq_hash      int32\n",
       "tw_first_word     int32\n",
       "tw_second_word    int32\n",
       "tw_last_word      int32\n",
       "tw_llast_word     int32\n",
       "tw_len            int32\n",
       "tw_hash0          int32\n",
       "tw_hash1          int32\n",
       "tw_rt_uhash       int32\n",
       "dtype: object"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DF.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
